# -*- coding: utf-8 -*-
'''
Created on Aug 4, 2012

@author: LONG HOANG GIANG
'''

import sys, os
import traceback
import time
sys.path.append(os.path.expanduser('/home5/vietcntt/longhoanggiang/python'))
sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
from CrawlerLib import Http, commonlib, html2text
import datetime
import re
from urlparse import urljoin
from lxml import etree
import sqlite3 as db

def isExistsStory(item, data):
    for i in data:
        if item['title'] == i['title']: return True
    return False

def getListStory(url):
    print '>start get list story'
#    url = 'http://vnthuquan.net/truyen/truyen.aspx?tid=2qtqv3m3237nmnnntn31n343tq83a3q3m3237nvn&AspxAutoDetectCookieSupport=1'
    tree = commonlib.loadweb(url).build_tree()
    data = []
    for item in tree.xpath("//td[.='Mục Lục']/../../following-sibling::*[1]/tr//acronym/a".decode('utf-8')):
        title = commonlib.stringify(item)
        link = item.get('href')
        link = urljoin('http://vnthuquan.net/truyen/', link)
        ii = {'title': title, 'url': link}
        if not isExistsStory(ii, data): data.append(ii)
    return data

def createTable(c):
    c.execute("DROP TABLE IF EXISTS tblChapters")
    c.execute("DROP TABLE IF EXISTS tblDetail")
    c.execute("CREATE TABLE tblChapters(`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `title` TEXT)")
    c.execute("CREATE INDEX chapter_idx ON tblChapters(id)")
    c.execute("CREATE TABLE tblDetail(`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `chapterId` INTEGER, `detail` TEXT)")
    c.execute("CREATE INDEX detail_idx ON tblDetail(chapterId)")

def getContent(item):
    url = item['url']
    print '------- get content {0}: {1}'.format(item['title'], url)
    for itime in range(0, 10):
        try:
            tree = commonlib.loadweb(url).build_tree()
            contentElement = tree.xpath("//div[@id='fontchu' and @class='truyen_text']/preceding-sibling::*[1]")[0]
            pNode = contentElement.getparent()
            pNode.remove(contentElement)
            content = html2text.html2text(commonlib.Etree.tohtml(pNode), url, download=commonlib.download_image, path='/hehe')
            print content
            content = re.sub(r"\n", "<br />", content)
            content = '''<h4>{0}</h4><br /><br />{1}'''.format(item['title'], content)
            break
        except:
            traceback.print_exc()
            itime += 1
            time.sleep(5)
    return content

def process():
    
    conn = db.connect(DBName)
    c = conn.cursor()
    createTable(c)
    
    links = getListStory(URL)
    
    for item in links:
        detail = getContent(item)
        detail = detail.decode('utf-8')
        if detail.strip() == '': continue
        c.execute("INSERT INTO tblChapters(title) VALUES(?)", [item['title'].decode('utf-8')])
        chapterId = c.lastrowid
        c.execute("INSERT INTO tblDetail(chapterId, detail) VALUES(?, ?)", (chapterId, detail))
    
    c.close()
    conn.commit()
    conn.close()

if __name__ == '__main__':
    ##################################################
    DBName = 'c:/Users/LONG HOANG GIANG/Desktop/story'
    URL = 'http://vnthuquan.net/truyen/truyen.aspx?tid=2qtqv3m3237n2nmn4nnn31n343tq83a3q3m3237nvn'
    ##################################################
    process()
    print '> Finished at {0}'.format(datetime.datetime.now())
    os._exit(1)
    
